#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :lvmama.py
# @Time      :2023/10/25
# @Author    :CL
# @email     :1037654919@qq.com
import re

import requests
from bs4 import BeautifulSoup
from retrying import retry
from utils import mongo_manager, get_kuai_proxy

lvmama_sight_info = mongo_manager('lvmama_sight_info', db='lvyou')

class Lvmama():
    def __init__(self):
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Pragma": "no-cache",
            "Proxy-Connection": "keep-alive",
            "Referer": "http://www.lvmama.com/lvyou/",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
        }
        self.cookies = {
            "_ip_city_name": "%E4%B8%8A%E6%B5%B7",
            "_ip_city_place_id": "310000",
            "_ip_province_place_id": "310000",
            "uid": "wKgKb2U4c6DB6FY8N2IDAg==",
            "lvsessionid": "5e91e5ff-40d7-44d8-80c7-93b29dc8e917_16864194",
            "BDCCOOKIEID": "hrBhePbj3980950582",
            "BDCSESSIONID": "rSjdsWfeSHy1698217649443",
            "CoreID6": "88359017514216982176497&ci=52710000|PC",
            "Hm_lvt_cb09ebb4692b521604e77f4bf0a61013": "1698217650",
            "fp_ver": "4.5.1",
            "_ga": "GA1.2.481363095.1698217651",
            "_gid": "GA1.2.875696478.1698217651",
            "_jzqc": "1",
            "PHPSESSID": "21rbooo9huibt9r0drchq3h2h0",
            "cmTPSet": "Y",
            "_lvTrack_u_ud": "B014A4D3-920F-4CA4-A91C-ACAE28A90E01",
            "_qzjc": "1",
            "LVMM_NODE_ENV": "production",
            "oIC": "233198233198233198233198",
            "oIT": "0925092509250925",
            "_jzqckmp": "1",
            "_lvTrack_u_sd": "96837417-21F0-49C2-90C0-2A001AC2A6BA",
            "_lvTrack_firstVisitTime": "1698306676715",
            "_lvTrack_preVisitTime": "1698306676718",
            "__xsptplus443": "443.3.1698306806.1698306806.1%234%7C%7C%7C%7C%7C%23%23%23",
            "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2218b6483ae6e583-00a9889c6d2904-13462c6c-2073600-18b6483ae6f94b%22%2C%22%24device_id%22%3A%2218b6483ae6e583-00a9889c6d2904-13462c6c-2073600-18b6483ae6f94b%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E5%BC%95%E8%8D%90%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fblog.csdn.net%2Fqq_33854211%2Farticle%2Fdetails%2F78766621%3Fspm%3D1001.2101.3001.6650.2%26utm_medium%3Ddistribute.pc_relevant.none-task-blog-2%257Edefault%257ECTRLIST%257ERate-2-78766621-blog-129261872.235%255Ev38%255%22%2C%22%24latest_referrer_host%22%3A%22blog.csdn.net%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%7D%7D",
            "_qzja": "1.402745185.1698218091536.1698306531850.1698312931857.1698309180491.1698312931857.0.0.0.35.5",
            "52710000|PC_clogin": "v=37&l=1698368303&e=1698370109850",
            "Hm_lpvt_cb09ebb4692b521604e77f4bf0a61013": "1698368310",
            "_jzqa": "1.478980885840175800.1698217652.1698312932.1698368310.6",
            "_jzqx": "1.1698299990.1698368310.3.jzqsr=lvmama%2Ecom|jzqct=/lvyou/d-beijing1%2Ehtml.jzqsr=lvmama%2Ecom|jzqct=/",
            "_jzqb": "1.1.10.1698368310.1",
            "_gat": "1",
            "_ga_NHEPS49S4W": "GS1.2.1698368312.6.0.1698368312.0.0.0",
            "_fmdata": "Lgnmwvb9UZV3dOw7baJRkFLIcqZ0qBmuoEw2PbzMKQxBloJQrAqlHMCiqafNRvHd8lh1U64CWgAXqAfbf%2B%2Fn25K9h%2FPATJCX5Kw51ZOsDq8%3D",
            "JSESSIONID": "C0DFA01B9A12CC1861911896B94A5801"
        }
        self.proxy = get_kuai_proxy()

    # 获取目的地href
    @retry(stop_max_attempt_number=3)
    def get_mudidi(self):
        url = "http://www.lvmama.com/lvyou/"
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxy, timeout=10,
                                verify=False)
        # print(response.text)
        # print(response)
        soups = BeautifulSoup(response.text, 'lxml')
        datas = soups.find('div', class_='wy_liNav_box wy_liNav_boxThree').find_all('dl')
        # print(datas)
        result = []
        for data in datas:
            topic = data.find('dt').get_text()
            hrefs = [[a.get_text(), '/lvyou/scenery/' + a['href']] for a in data.find_all('a')]
            # print(topic, hrefs)
            result.append([topic, hrefs])
        return result

    @retry(stop_max_attempt_number=5)
    # 基于目的地获取景点列表
    def get_viewlist(self, data=None):

        url = "http://www.lvmama.com/lvyou/ajax/getNewViewList"
        response = requests.post(url, headers=self.headers, cookies=self.cookies, data=data, timeout=20, verify=False)
        requests.session().close()
        # print(response.text)
        print(response)
        if response.status_code == 200:
            # print(response.json()['data'])
            res = response.json()['data']
            return res
    @retry(stop_max_attempt_number=3)
    def get_sight_info(self, url=None):
        headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "no-cache",
            "Pragma": "no-cache",
            "Proxy-Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
        }
        cookies = {
            "_ip_city_name": "%E4%B8%8A%E6%B5%B7",
            "_ip_city_place_id": "310000",
            "_ip_province_place_id": "310000",
            "uid": "wKgKb2U4c6DB6FY8N2IDAg==",
            "lvsessionid": "5e91e5ff-40d7-44d8-80c7-93b29dc8e917_16864194",
            "BDCCOOKIEID": "hrBhePbj3980950582",
            "BDCSESSIONID": "rSjdsWfeSHy1698217649443",
            "CoreID6": "88359017514216982176497&ci=52710000|PC",
            "Hm_lvt_cb09ebb4692b521604e77f4bf0a61013": "1698217650",
            "fp_ver": "4.5.1",
            "_ga": "GA1.2.481363095.1698217651",
            "_gid": "GA1.2.875696478.1698217651",
            "_jzqc": "1",
            "PHPSESSID": "21rbooo9huibt9r0drchq3h2h0",
            "cmTPSet": "Y",
            "_lvTrack_u_ud": "B014A4D3-920F-4CA4-A91C-ACAE28A90E01",
            "_qzjc": "1",
            "LVMM_NODE_ENV": "production",
            "oIC": "233198233198233198233198",
            "oIT": "0925092509250925",
            "_jzqckmp": "1",
            "_lvTrack_u_sd": "96837417-21F0-49C2-90C0-2A001AC2A6BA",
            "_lvTrack_firstVisitTime": "1698306676715",
            "_lvTrack_preVisitTime": "1698306676718",
            "__xsptplus443": "443.3.1698306806.1698306806.1%234%7C%7C%7C%7C%7C%23%23%23",
            "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2218b6483ae6e583-00a9889c6d2904-13462c6c-2073600-18b6483ae6f94b%22%2C%22%24device_id%22%3A%2218b6483ae6e583-00a9889c6d2904-13462c6c-2073600-18b6483ae6f94b%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E5%BC%95%E8%8D%90%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fblog.csdn.net%2Fqq_33854211%2Farticle%2Fdetails%2F78766621%3Fspm%3D1001.2101.3001.6650.2%26utm_medium%3Ddistribute.pc_relevant.none-task-blog-2%257Edefault%257ECTRLIST%257ERate-2-78766621-blog-129261872.235%255Ev38%255%22%2C%22%24latest_referrer_host%22%3A%22blog.csdn.net%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%7D%7D",
            "_qzja": "1.402745185.1698218091536.1698306531850.1698312931857.1698309180491.1698312931857.0.0.0.35.5",
            "52710000|PC_clogin": "v=37&l=1698368303&e=1698370109850",
            "Hm_lpvt_cb09ebb4692b521604e77f4bf0a61013": "1698368310",
            "_jzqa": "1.478980885840175800.1698217652.1698312932.1698368310.6",
            "_jzqx": "1.1698299990.1698368310.3.jzqsr=lvmama%2Ecom|jzqct=/lvyou/d-beijing1%2Ehtml.jzqsr=lvmama%2Ecom|jzqct=/",
            "_jzqb": "1.1.10.1698368310.1",
            "_gat": "1",
            "_ga_NHEPS49S4W": "GS1.2.1698368312.6.0.1698368312.0.0.0",
            "_fmdata": "Lgnmwvb9UZV3dOw7baJRkFLIcqZ0qBmuoEw2PbzMKQxBloJQrAqlHMCiqafNRvHd8lh1U64CWgAXqAfbf%2B%2Fn25K9h%2FPATJCX5Kw51ZOsDq8%3D",
            "JSESSIONID": "C0DFA01B9A12CC1861911896B94A5801"
        }
        if url == None:
            url = "http://www.lvmama.com/lvyou/poi/sight-175076.html"

        response = requests.get(url, headers=headers, cookies=cookies, timeout=20, verify=False)
        requests.session().close()
        # print(response.text)
        print(response)
        if response.status_code == 200:
            return response.text

    @retry
    def get_scenery(self, url):

        response = requests.get(url, headers=self.headers, cookies=self.cookies, timeout=20, verify=False)
        # print(response)
        if response.status_code == 200:
            return response.text


def save_viewlist(res, basehref):
    soups = BeautifulSoup(res, 'lxml')
    datas = soups.find_all('dl')
    for data in datas:
        img = data.find('dt').find('img').get('src')
        poi_name = data.find('dd').find('div', class_='title').get_text()
        href = data.find('dd').find('div', class_='title').find('a').get('href')
        try:
            tagbox = data.find('dd').find('div', class_='poi-tagbox').get_text()
        except:
            tagbox = ''
        poi_per_txt = ''
        try:
            poi_per = data.find('dd').find_all('p', class_='poi-per-com1')
            for paragraph in poi_per:
                poi_per_txt += paragraph.get_text() + '\n'
        except:
            pass

        try:
            ticket = data.find('dd').find('div', class_='iLike').get_text()
        except:
            ticket = ''
        # print(img,poi_name,href,tagbox,poi_per_txt,ticket)
        result = {"_id": href, "poi_name": poi_name, "href": href, "img": img, 'basehref': basehref,
                  "tagbox": tagbox, 'poi_per_txt': poi_per_txt, 'ticket': ticket
                  }
        try:

            lvmama_sight_info.insertOne(result)
        except  BaseException as e:
            # lvmama_sight_info.updateOne({"_id":result["_id"]},result)
            print(e)

    pass


def save_mudidi():
    mudidilist = lvmama.get_mudidi()
    # print(mudidilist)
    lvmama_mudidi = mongo_manager('lvmama_mudidi', db='lvyou')
    for topic in mudidilist:
        print(topic[0])
        for mudidi in topic[1]:
            name = mudidi[0]
            href = mudidi[1]
            try:
                data = {'_id': href, 'topic': topic, 'name': name, 'href': href}
                lvmama_mudidi.insertOne(data)
            except  BaseException as e:
                print(e)


def main():
    lvmama_mudidi = mongo_manager('lvmama_mudidi', db='lvyou')
    # lvmama_mudidi.updateMany({'status':'success'},{'status':None})
    seeds = lvmama_mudidi.findAll({'status': None, 'base_id': {'$exists': True}})
    lists = []
    for seed in seeds:
        lists.append(seed)
    print(len(lists))
    for seed in lists:
        print(seed)
        href = seed['_id']
        if href == '/lvyou/scenery/d-shaoxing111.html':
            seed['status'] = 'success'
            lvmama_mudidi.updateOne({"_id": seed["_id"]}, seed)
            continue
        con_list=['/lvyou/scenery/d-anji110.html']
        if href in con_list:
            continue
        dest_id = seed['dest_id']
        base_id = seed['base_id']
        baseurl = seed['href']
        data = {
            "page_num": "1",
            "dest_id": dest_id,
            "base_id": base_id,
            "request_uri": href
        }
        try:
            res = lvmama.get_viewlist(data)
            # print(f'res:{res}')
            save_viewlist(res, baseurl)
            wy_state_page = BeautifulSoup(res, 'lxml').find('div', class_='wy_state_page')
            total_num = wy_state_page.find('span', id='total_num').get_text()
            print(href,total_num)
            for page in range(2, int(int(total_num) / 10) + 1):
                print(f'{href},page:{page}')
                data = {
                    "page_num": page,
                    "dest_id": dest_id,
                    "base_id": base_id,
                    "request_uri": href
                }
                res = lvmama.get_viewlist(data)
                # print(f'res:{res}')
                save_viewlist(res, baseurl)
            seed['status'] = 'success'
            lvmama_mudidi.updateOne({"_id": seed["_id"]}, seed)
        except BaseException as e:
            print(e)
            seed['status'] = 'fail'
            lvmama_mudidi.updateOne({"_id": seed["_id"]}, seed)
    lvmama_mudidi.close()


if __name__ == '__main__':
    lvmama = Lvmama()
    main()

